﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using RCloud.Data;
using System.Threading;
using System.Runtime.Serialization.Formatters.Binary;
using System.IO;
using System.Linq.Expressions;
using System.Data.Common;
using System.Data.Objects.DataClasses;
using System.Collections;
using System.Runtime.Serialization;

/*
 * 思考：是否需要概化？如果概化是否会导致某个关键字【如类别】长久的霸占位置。
 * 目前模型的缺点：一旦一个类别关键字成功占据显要位置，那么此关键字将长期处于领先位置。且其它关键字均处于极小状态
 * 解决办法：取前20个关键字？然后至少为1？
 * 对小关键字的解决办法：
 */

[assembly: EdmSchemaAttribute()]
namespace RCloud.Windows
{
    public partial class Form1 : Form
    {
        Model model = null;

        public Form1()
        {
            InitializeComponent();
        }

        private void Form1_Load(object sender, EventArgs e)
        {
            RCloud.Data.REntitiesProvider.InitEntities(new RCloud.Data.REntitiesProvider());
        }

        public void NewTip(string msg)
        {
            processTip.Text = msg;
            resultListBox.Items.Add(string.Format("{0:hh:MM:ss}:{1}", DateTime.Now, msg));
        }

        public string DataPath
        {
            get
            {
                return @"d:\DB\txt\";
            }
        }

        const int tickEn = 830;

        const int tickStep = 1000000;

        /// <summary>
        /// 是否允许遗忘
        /// </summary>
        const bool allowForget = false;

        /// <summary>
        /// 遗忘因子
        /// </summary>
        const double forgetYZ = 0.2;

        int TimeStamp
        {
            get
            {
                return tickEn * 1000000;
            }
        }

        #region 调试阶段

        private void button1_Click(object sender, EventArgs e)
        {
            NewTip("开始处理数据");
            new Thread(new ParameterizedThreadStart(ConvertRating)).Start(tickEn);
            //ConvertRating(tickEn);
            button1.Enabled = false;
        }

        int step;
        int _mi;

        void SetProgressMax(int max)
        {
            dataProgress.Maximum = 100;
            dataProgress.Value = 0;
            step = max / 100;
            _mi = step;
            Thread.Yield();
        }

        void IncreaseProgress()
        {
            _mi--;
            if (_mi == 0)
            {
                if (dataProgress.Value < 100) dataProgress.Value++;
                _mi = step;
                Thread.Yield();
            }
        }


        #region 处理数据并将其转换成用户偏好、商品标签权重等

        /// <summary>
        /// 处理数据到某个时间点
        /// </summary>
        public void ConvertRating(object tickEnObj)
        {
            int tickEn = (int)tickEnObj;
            NewTip("读取投票数据中...");
            //1.读取所有数据
            Data.List.RatingHistory rhClass = new Data.List.RatingHistory { RowCount = int.MaxValue, TimeStampEnd = TimeStamp };
            var hisdata = rhClass.GetArray();
            NewTip("完成");

            //2.准备电影标签初始数据
            Dictionary<long, UserData> users = new Dictionary<long, UserData>();
            Dictionary<long, MovieData> movies = new Dictionary<long, MovieData>();

            NewTip("加载电影类型");
            NewTip("加载完毕，共有" + LoadMovieGenres(movies) + "条记录");

            //NewTip("加载电影标签");
            //NewTip("加载完毕，共有" + LoadMovieTags(movies) + "条记录");

            //3.开始正式处理
            NewTip("开始处理投票数据，共" + hisdata.Length + "条");
            SetProgressMax(hisdata.Length);
            foreach (var his in hisdata)
            {
                MovieData md = movies[his.MovieID];
                UserData ud;
                long uid = his.UserID;
                if (!users.TryGetValue(uid, out ud))
                {
                    ud = new UserData();
                    users.Add(uid, ud);
                }
                int rating = (int)his.Rating - 4;//负分处理

                //3.1先对用户处理
                ud.RateMovie(md, rating);

                //3.2再对电影处理
                md.BeRated(ud, rating);

                IncreaseProgress();

            }
            NewTip("处理完毕，处理了" + users.Count + "个用户");
            hisdata = null;
            GC.Collect();

            //4.处理完毕，将结果保存到数据库
            NewTip("保存历史记录");

            /*string fnp1 = "tagresult" + tickEn;
            FileStream fileStream = new FileStream(DataPath + fnp1 + "mv", FileMode.Create);
            BinaryFormatter b = new BinaryFormatter();
            b.Serialize(fileStream, movies);
            fileStream.Close();

            fileStream = new FileStream(DataPath + fnp1 + "usr", FileMode.Create);
            b.Serialize(fileStream, users);
            fileStream.Close();

            */
            NewTip("保存完毕");

            Movies = movies;
            Users = users;

            /*
            NewTip("保存到数据库");
            Data.MovieTagsNewUpdateClass mtuClass = new MovieTagsNewUpdateClass { Mode = Doo.DataClass.UpdateMode.OnlyInsert };
            using (var container = mtuClass.GetEntities())
            {
                SetProgressMax(movies.Count());
                foreach (KeyValuePair<long, MovieData> kv in movies)
                {
                    mtuClass.MovieID = kv.Key;
                    foreach (KeyValuePair<string, PItem> rate in kv.Value)
                    {
                        mtuClass.Tag = rate.Key;
                        mtuClass.UpdateFunc = r => { r.SumCount = rate.Value.SumCount; return true; };
                        mtuClass.Process(container, false);
                    }
                    container.SaveChanges();
                    IncreaseProgress();
                }
            }
            NewTip("保存成功");
             */
        }

        #endregion

        /// <summary>
        /// 标签辞典
        /// </summary>
        public Dictionary<int, string> TagDict = new Dictionary<int, string>();

        public Dictionary<string, int> TagToIndex = new Dictionary<string, int>();

        public int MaxIndex = 0;

        int GetTagIndex(string tag)
        {
            if (TagToIndex.ContainsKey(tag)) return TagToIndex[tag];
            MaxIndex++;
            TagToIndex.Add(tag, MaxIndex);
            return MaxIndex;
        }

        class PData : Dictionary<long, int>
        {
            public PData() : base() { }
            const string ser_Max = "M";
            const string ser_Min = "I";
            const string ser_Avg = "A";
            const string ser_TopTags = "T";
            /// <summary>
            /// 遗忘因子
            /// </summary>
            const double cst_forget = forgetYZ;

            protected PData(SerializationInfo info, StreamingContext context)
                : base(info, context)
            {
                Max = info.GetInt32(ser_Max);
                //Avg = info.GetInt32(ser_Avg);
                Avg = (int)(Max * 0.6);
                TopTags = info.GetValue(ser_TopTags, typeof(Dictionary<long, int>)) as Dictionary<long, int>;
                TopRank.Clear();
                foreach (var k in TopTags.Keys)
                {
                    TopRank.Add(this[k]);
                }
            }

            public override void GetObjectData(SerializationInfo info, StreamingContext context)
            {
                info.AddValue(ser_Max, Max);
                //info.AddValue(ser_Avg, Avg);
                info.AddValue(ser_TopTags, cachedTags);
                base.GetObjectData(info, context);
            }

            /// <summary>
            /// 前一次的记录，用于遗忘
            /// </summary>
            Dictionary<long, int> PrevData = null;

            Dictionary<long, int> VisitTotal = new Dictionary<long, int>();

            /// <summary>
            /// 执行一次遗忘
            /// </summary>
            public void Foreget()
            {
                if (PrevData != null)
                {
                    var pd = PrevData;
                    foreach (var kv in pd)
                    {
                        int add = this[kv.Key];
                        int newnew = (int)(kv.Value * cst_forget + (add - kv.Value));
                        this[kv.Key] = newnew;

                        if (add >= TopRank.Min && add != newnew)
                        {
                            TopRank.Remove(add);
                            if (newnew > TopRank.Min)
                            {
                                TopRank.Add(newnew);
                            }
                        }
                        else if (newnew > TopRank.Min)
                        {
                            TopRank.Add(newnew);
                            if (!cachedTags.ContainsKey(kv.Key)) cachedTags.Add(kv.Key, 0);
                        }
                    }
                    needRefresh = true;
                    EndProcessTag();
                }

                PrevData = new Dictionary<long, int>(this);
            }

            int Max = 1;

            /// <summary>
            /// 及格线
            /// </summary>
            int Avg = 0;

            /// <summary>
            /// 最讨厌的
            /// </summary>
            int Min = 0;

            /// <summary>
            /// 前RankMax排位
            /// </summary>
            SortedSet<int> TopRank = new SortedSet<int>(Comparer<int>.Default);

            const int RankMax = 30;

            /// <summary>
            /// 已缓存的有效偏好
            /// </summary>
            Dictionary<long, int> cachedTags = new Dictionary<long, int>();

            /// <summary>
            /// 取出已概化为1-5分的有效偏好
            /// </summary>
            /// <returns></returns>
            public Dictionary<long, int> TopTags
            {
                get { return cachedTags; }
                set { cachedTags = value; }
            }

            /// <summary>
            /// 不喜欢的标签集合
            /// </summary>
            Dictionary<long, int> hateTags = new Dictionary<long, int>();

            public Dictionary<long, int> HateTags
            {
                get { return hateTags; }
            }

            public int ProcessCount = 0;

            bool needRefresh = false;

            /// <summary>
            /// 准备工作
            /// </summary>
            public void BeginProcessTag()
            {
                //暂时没有
            }

            /// <summary>
            /// 做完了ProcessTag需要做一些收尾工作
            /// 在Tag被其它地方使用前必须处理此方法
            /// </summary>
            public void EndProcessTag()
            {
                if (needRefresh)
                {
                    cachedTags = (from t in cachedTags
                                  orderby this[t.Key] descending
                                  select t)
                                  .Take(20)
                                  .ToDictionary(t => t.Key, t => ((int)((this[t.Key] - 1) * 10 / Max) + 1));

                    hateTags = (from t in hateTags
                                orderby this[t.Key]
                                select t)
                                  .Take(20)
                                  .ToDictionary(t => t.Key, t => ((int)((this[t.Key] + 1) * 10 / Min)));//负负得正

                    needRefresh = false;
                }
                ProcessCount++;
            }

            /// <summary>
            /// 处理标签评分，将评分自增
            /// </summary>
            /// <param name="name">一个索引，指向TagDict的某一点</param>
            /// <param name="score"></param>
            public void ProcessTag(long name, int rating, int max)
            {
                //如果评分为0，则没什么可做的
                if (rating == 0) return;

                //处理VisitTotal
                int total;
                int tmp = max;
                int score;
                int oldscore = 0;
                int oldtotal = 1;
                if (VisitTotal.TryGetValue(name, out total))
                {
                    oldtotal = total;
                    total += tmp;
                    VisitTotal[name] = total;

                    oldscore = this[name];
                    score = rating + oldscore;
                    this[name] = score;
                }
                else
                {
                    total = tmp;
                    VisitTotal.Add(name, total);

                    score = rating;
                    Add(name, rating);
                }

                //if (total < 120 || score < 60) return;//低于120不予处理

                int 比例 = score * 10 / total;
                int oldPer = oldscore * 10 / oldtotal;

                if (oldPer != 比例)
                {
                    bool joinTop = false, joinHate = false;
                    int i = 0;
                    if (比例 > 5)
                    {
                        joinTop = true;
                        i = (比例 - 5) * 2;
                    }
                    else if (比例 < 3)
                    {
                        joinHate = true;
                        i = (5 - 比例) * 2;
                    }

                    var dict = joinTop ? TopTags : joinHate ? HateTags : null;
                    if (dict != null)
                    {
                        if (dict.ContainsKey(name))
                        {
                            dict[name] = i;
                        }
                        else
                        {
                            dict.Add(name, i);
                        }
                    }

                    if (!joinTop)
                    {
                        if (TopTags.ContainsKey(name)) TopTags.Remove(name);
                    }

                    if (!joinHate)
                    {
                        if (hateTags.ContainsKey(name)) hateTags.Remove(name);
                    }

                }
                return;

                #region 旧代码
                //是否越过最低线
                if (rating < 0)
                {
                    if (oldscore >= TopRank.Min)
                    {
                        TopRank.Remove(oldscore);
                        needRefresh = true;
                        if (score > TopRank.Min)
                        {
                            TopRank.Add(score);
                        }
                        else if (cachedTags.ContainsKey(name))
                        {
                            cachedTags.Remove(name);
                        }
                    }
                }
                else if (TopRank.Count < RankMax || score > TopRank.Min)
                {
                    if (oldscore >= TopRank.Min) TopRank.Remove(oldscore);
                    TopRank.Add(score);
                    if (TopRank.Count > RankMax) TopRank.Remove(TopRank.Min);

                    needRefresh = true;
                    if (!cachedTags.ContainsKey(name)) cachedTags.Add(name, 0);
                    if (score > Max)
                    {
                        Max = score;
                        Avg = (int)(Max * 0.6);
                    }
                }

                if (score > 0 && oldscore < 0 && hateTags.ContainsKey(name))
                {
                    hateTags.Remove(name);
                }

                if (score < 0 && oldscore >= 0 && !hateTags.ContainsKey(name))
                {
                    hateTags.Add(name, 0);

                    if (score < Min)
                    {
                        Min = score;
                    }

                    needRefresh = true;
                }

                #endregion
            }

            public int RateTime = 0;

            protected void Rate(PData pd, int rate)
            {
                RateTime++;

                //取出影片中有效偏好，对自己的数据进行修正
                var mvtags = pd.TopTags;
                BeginProcessTag();
                foreach (KeyValuePair<long, int> kv in mvtags.Union(pd.HateTags))
                {
                    //依据用户和商品的权值确定应该增加的权值
                    //此处评分分正负两种，正分表示喜欢，负分表示讨厌

                    int ps = kv.Value;//商品
                    /*int ms;//我的
                    if (!cachedTags.TryGetValue(kv.Key, out ms))
                    {
                        ms = 0;
                    }

                    int newrate = 0;
                    //正分表示喜欢，从而为匹配的标签加上权值
                    if (rate > 0)
                    {
                        newrate = ps * rate;
                    }
                    else
                    {
                        //负分表示讨厌，从而降低标签的权值
                        newrate = ps * rate;
                    }*/
                    //int tmp1 = 10 + (int)((ps - ms) / 3);//特征差
                    //int tmp2 = (int)Math.Sqrt(11 - ms);//距离满分差

                    //int newrate = rate > 0 ? (int)tmp1 * rate : ps * rate;
                    //if (ps > 5 && ms > 5) throw new Exception();

                    int newrate = ps * rate;
                    ProcessTag(kv.Key, newrate, 5 * ps);//此处rate最大值是6
                }
                EndProcessTag();
            }
        }

        /// <summary>
        /// 用户数据
        /// </summary>
        [Serializable]
        class UserData : PData
        {
            public UserData() : base() { }

            protected UserData(SerializationInfo info, StreamingContext context)
                : base(info, context)
            { }

            /// <summary>
            /// 为某个影片做了投票，更新自己的偏好数据库
            /// </summary>
            /// <param name="mv"></param>
            public void RateMovie(MovieData mv, int rate)
            {
                base.Rate(mv, rate);
            }
        }

        [Serializable]
        class MovieData : PData
        {

            public MovieData() : base() { }

            protected MovieData(SerializationInfo info, StreamingContext context)
                : base(info, context)
            { }

            /// <summary>
            /// 被人投票，更新自己的偏好数据库
            /// </summary>
            /// <param name="mv"></param>
            public void BeRated(UserData ud, int rate)
            {
                RateTime++;

                var mvtags = ud.TopTags;
                BeginProcessTag();
                foreach (KeyValuePair<long, int> kv in mvtags)
                {
                    int up = kv.Value;//用户的评分
                    int mp;

                    TopTags.TryGetValue(kv.Key, out mp);

                    int newrate = rate;
                    newrate *= up;
                    //if (rate < 0) newrate = -newrate;
                    ProcessTag(kv.Key, newrate, 5 * kv.Value);
                }
                EndProcessTag();
            }
        }

        private void button2_Click(object sender, EventArgs e)
        {
            ThreadStart ts = new ThreadStart(ForecaseNew);
            new Thread(ts).Start();
        }

        void ForecaseNew()
        {
            EnsureModel();

            Data.List.RatingHistory rhClass = new Data.List.RatingHistory { RowCount = 5000, TimeStampStart = TimeStamp };
            var hisdata = rhClass.GetArray();

            foreach (var h in hisdata)
            {
                var result = model.ForestRating(h.UserID, h.MovieID);
                int foreScore = result.Rating;
                int wucha = foreScore - (int)h.Rating;
                int abs = Math.Abs(wucha);

                dataGridForeCast.Rows.Add(h.UserID, h.MovieID, foreScore, h.Rating, wucha, Math.Abs(wucha), 0, 0);


            }
        }

        Dictionary<long, UserData> _users;

        Dictionary<long, UserData> Users
        {
            get
            {
                if (_users == null)
                {
                    string fnp1 = "tagresult" + tickEn;
                    FileStream fileStream = new FileStream(DataPath + fnp1 + "usr", FileMode.Open);
                    BinaryFormatter b = new BinaryFormatter();
                    _users = b.Deserialize(fileStream) as Dictionary<long, UserData>;
                    fileStream.Close();
                }
                return _users;
            }
            set
            {
                _users = value;
            }
        }
        Dictionary<long, MovieData> _movies;

        Dictionary<long, MovieData> Movies
        {
            get
            {
                if (_movies == null)
                {
                    string fnp1 = "tagresult" + tickEn;
                    FileStream fileStream = new FileStream(DataPath + fnp1 + "mv", FileMode.Open);
                    BinaryFormatter b = new BinaryFormatter();
                    _movies = b.Deserialize(fileStream) as Dictionary<long, MovieData>;
                    fileStream.Close();
                }
                return _movies;
            }
            set
            {
                _movies = value;
            }
        }

        #region 检查算法是否正确

        public void SetForecastTip(string tip)
        {
            labelForecast.Text = tip;
            txtTip.Text += "\r\n" + tip;
        }

        class ForecastRatingReturn
        {
            public int match;
            public int score;
            public string detail;
        }

        static double q2 = Math.Sqrt(2d);

        /// <summary>
        /// 预测用户对电影的评分
        /// </summary>
        /// <param name="user"></param>
        /// <param name="movie"></param>
        /// <returns></returns>
        ForecastRatingReturn ForecastRating(UserData ud, MovieData md, bool needdetail)
        {
            //检查user的tags是否与md的tags对应
            var mts = md.TopTags;
            var mht = md.HateTags;
            var uts = ud.TopTags;
            var uht = ud.HateTags;
            int umax = uts.Count;//用户标签的数量
            double score = 0;
            StringBuilder sb = null;
            if (needdetail)
            {
                sb = new StringBuilder();
                sb.AppendFormat("用户投票数{0}电影被投数{1}用户偏好数{2}电影标签数{3}#", ud.ProcessCount, md.ProcessCount, uts.Count, mts.Count);
            }
            double match = 0;

            //比较方法
            //针对自己喜欢的特征，如对方特征中出现，加分
            //针对自己讨厌的特征，如对方特征中出现，减分
            //针对没有的特征在对方特征中出现的，要考虑

            foreach (var ui in uts)
            {
                //好特征
                int us = ui.Value;
                if (us > 10) throw new Exception();
                //查对方好特征
                int ps;
                if (mts.TryGetValue(us, out ps))
                {
                    //好特征，加分
                    match++;
                    int m2 = ps * ps;
                    int u2 = us * us;
                    score += Math.Sqrt((2 * m2 * u2 / (u2 + m2)));
                }
            }

            foreach (var ui in uht)
            {
                int us = ui.Value;
                int ps;
                if (mts.TryGetValue(us, out ps))
                {
                    match++;
                    int m2 = ps * ps;
                    int u2 = us * us;
                    score -= Math.Sqrt((2 * m2 * u2 / (u2 + m2)));
                }
            }

            int newmatch = 0, newscore = 0;

            foreach (var mi in mts)
            {
                if (uts.ContainsKey(mi.Key) || uht.ContainsKey(mi.Key)) continue;
                newscore += mi.Value;//作为少量考虑
                newmatch++;
            }

            foreach (var mi in mht)
            {
                if (uts.ContainsKey(mi.Key) || uht.ContainsKey(mi.Key)) continue;
                newscore -= mi.Value;
                newmatch++;
            }

            int offset = newmatch == 0 ? 0 : (newscore / 3 / newmatch);//可以产生的影响

            int foreScore = 5;
            if (match > 0) foreScore += (int)(score / (2 * match)) + offset;

            if (foreScore > 10) foreScore = 10;
            else if (foreScore < 0) foreScore = 0;

            return new ForecastRatingReturn
            {
                score = foreScore,
                detail = sb == null ? null : sb.ToString(),
                match = (int)match
            };
            /*

            //用户标签与电影标签进行匹配
            //匹配值用直角三角中线长度
            //多匹配值的合计用【正方形面积累加法:R2】
            foreach (var ui in uts)
            {
                int mi = 0;
                int uv = ui.Value;
                int u2 = uv * uv;
                if (mts.TryGetValue(ui.Key, out mi))
                {
                    //已经知道两种评分
                    //1.mi：电影中此标签的权值【标签在所有投票人心中的权值】
                    //2.ui：用户偏好中此标签的权值
                    match++;

                    int m2 = mi * mi;

                    double tagScore = 0;
                    //1.使用直角三角形斜边作为tagScore
                    if (false)
                    {
                        tagScore =
                            (u2 + m2) / 4;
                        score += tagScore;
                    }
                    //2.使用斜边的垂线作为tagScore
                    else if (false)
                    {
                        tagScore =
                            (2 * m2 * u2 / (u2 + m2));//结果是0-10的数，允许有偏高
                        score += tagScore;
                    }
                    //3.使用特征相似度
                    else if (true)
                    {
                        tagScore = mi * u2;//自己的权值占比重大一点
                        score += tagScore;
                    }

                    if (needdetail)
                    {
                        sb.Append(ui.Value + "_" + mi + "_" + tagScore + "#  ");
                    }
                }
                else if (mht.ContainsKey(ui.Key))//包含讨厌的权值
                {
                    match++;
                    score -= mht[ui.Key] * u2;
                }
                else
                {
                    //SetForecastTip(ui.Key + "");
                    //score += ui.Value;
                    //没有的标签使用平均评分
                    //score += ui.Value * ui.Value;
                    //score += 5 * u2;
                }
            }

            //预测分数：开平方
            //score = Math.Sqrt(score);
            //score = Math.Sqrt(score) * 0.9;
            //score = new Random((int)DateTime.Now.Ticks).Next(1, 11);
            score = Math.Pow(score / match, 1d / 3);
            int foreScore = 5 + (int)(score / 2);// (int)(score * score);
            if (foreScore > 10) foreScore = 10;
            else if (foreScore < 0) foreScore = 0;

            return new ForecastRatingReturn
                {
                    score = foreScore,
                    detail = sb == null ? null : sb.ToString(),
                    match = match
                };*/
        }

        /// <summary>
        /// 预测用户最喜爱的前10部电影
        /// </summary>
        /// <param name="ud">用户</param>
        /// <returns></returns>
        long[] ForecastTopRating(UserData ud)
        {
            //算法：针对每个TopTag，取出所对应的最高评分的前十名
            var tgs = ud.TopTags;
            var topMovieClass = new Data.List.TopMovieClass { RowCount = 10 };
            Dictionary<long, int> processedMovie = new Dictionary<long, int>();
            int match = 0;

            foreach (var tag in tgs)
            {
                topMovieClass.TagID = tag.Key;
                var tMovies = topMovieClass.GetArray();
                foreach (var m in tMovies)
                {
                    if (processedMovie.ContainsKey(m.MovieID)) continue;
                    processedMovie.Add(m.MovieID, ForecastRating(ud, Movies[m.MovieID], false).score);
                }

            }
            //排序并返回前十名

            var query = from m in processedMovie
                        orderby m.Value descending
                        select m.Key;

            return query.Take(10).ToArray();
        }

        //判断标准A：
        //1、对下一条评价数据，由模型推算出可能的评分，然后和真实评分比较
        //2、推算依据：查找电影标签与用户标签的匹配度
        //  算法（使用相似度的算法？）：
        //3、报表：预测记录数、完全匹配数、相差1分、相差2分、相差3分
        //判断标准B:
        //1、对某个用户，判断其所喜好的前20部电影，查找真实评分
        //2、对没有匹配的真实评分做忽略处理，即此用户无效
        //3、报表同上
        public void Forecast()
        {
            //取出前100条记录来看看
            Data.List.RatingHistory rhClass = new Data.List.RatingHistory { RowCount = 5000, TimeStampStart = TimeStamp };
            Dictionary<long, UserData> users = new Dictionary<long, UserData>();
            Dictionary<long, MovieData> movies = new Dictionary<long, MovieData>();

            SetForecastTip("读取投票数据中...");

            var hisData = rhClass.GetArray();

            if (hisData.Length > 0)
            {
                SetForecastTip(string.Format("投票数据共有{0}条", hisData.Length));

                SetForecastTip("加载偏好数据中...");

                movies = Movies;

                SetForecastTip("电影标签数据加载完成");

                users = Users;

                SetForecastTip("用户标签数据加载完成");

            }
            else
            {
                SetForecastTip("没有数据");
            }

            int total = 0;
            int wu0 = 0;
            int wu1 = 0;
            int wu2 = 0;

            foreach (var h in hisData)
            {
                //1.预测
                MovieData md = movies[h.MovieID];
                if (!users.ContainsKey(h.UserID)) continue;

                total++;
                UserData ud = users[h.UserID];

                var value = ForecastRating(ud, md, true);
                int match = value.match;
                int foreScore = value.score;

                //2.检查是否命中
                int wucha = (int)h.Rating - foreScore;

                if (wucha == 0) wu0++;
                else if (wucha == 1) wu1++;
                else if (wucha == 2) wu2++;

                dataGridForeCast.Rows.Add(h.UserID, h.MovieID, foreScore, h.Rating, wucha, Math.Abs(wucha), match, value.detail);
            }

            txtTip.Text = string.Format("总计{0}，其中误差0={1}，误差0~1={2}({5}%)", total, wu0, wu1, wu2
                , (int)((wu0 + wu1 + wu2) * 100 / total)
                , (int)((wu0 + wu1) * 100 / total)
                );

            //依次查看
            /*
            string[] a = new string[] { "a", "bb" };

            //组建一个表达式树来创建一个参数
            ParameterExpression param =
                Expression.Parameter(typeof(string), "c");
            //组建表达式树:c.ContactName
            Expression selector = Expression.Property(param,
                typeof(string).GetProperty("Length"));

            using (var container = rhClass.GetEntities())
            {
                var q1 = container.CreateQuery<DbDataRecord>("select mt.MovieID as l from MovieTags as mt");

                var result = q1.Take(10).ToArray();

                var q2 = container.CreateQuery<DbDataRecord>("[MovieTags]")
                    .Where("it.MovieID>1234 and it.MovieID<1299")
                    .Include("[Movie] as mv")
                    .Where("mv.ID=it.MovieID")
                    .Select("it.MovieID,mv.Title");
                var result2 = q2.Take(10).ToList();

                //转换成Hashtable
                string[] fields = new string[] { "MovieID", "Tag" };

                List<Hashtable> lht = new List<Hashtable>(result2.Count);
                foreach (var r in result2)
                {
                    Hashtable ht = new Hashtable(fields.Length);
                    foreach (var f in fields)
                    {
                        ht[f] = r[f];
                    }
                    lht.Add(ht);
                }

                var str = new System.Web.Script.Serialization.JavaScriptSerializer().Serialize(lht);
            }
            Expression pred = Expression.Lambda(selector, param);
            Expression expr = Expression.Call(typeof(Queryable), "Select", new Type[] { typeof(string), typeof(int) }, Expression.Constant(a.AsQueryable()), pred);
            var q = a.AsQueryable().Provider.CreateQuery<int>(expr);
            var tmp = q.ToArray();
            MessageBox.Show("哈哈");
             */
        }

        #endregion

        void TopRating()
        {
            //在用户群中进行预测
            var users = Users;

            Data.List.UserRatingCheckClass urc = new Data.List.UserRatingCheckClass();

            using (var ent = urc.GetEntities())
            {
                foreach (var usr in users.Take(10))
                {
                    long uid = usr.Key;
                    var ud = usr.Value;
                    long[] mvs = ForecastTopRating(ud);
                    //这些mvs肯定是10分的，从数据库里面找看是不是十分的
                    StringBuilder sb = new StringBuilder();
                    int wucha = 0;
                    foreach (var mid in mvs)
                    {
                        long rating = urc.GetRating(ent, uid, mid);
                        wucha += (int)Math.Abs(rating - 10);
                        sb.AppendFormat("{2}\r\n", uid, mid, rating);
                    }
                    gridTopRating.Rows.Add(uid, mvs.Length, sb.ToString(), wucha);
                }
            }
        }

        private void btnStartTopRating_Click(object sender, EventArgs e)
        {
            ThreadStart ts = new ThreadStart(TopRating);
            new Thread(ts).Start();
        }

        private void btnProcess_Click(object sender, EventArgs e)
        {
            new Thread(new ThreadStart(ProcessNew)).Start();
            btnProcess.Enabled = false;
        }

        void ProcessTip(string tip)
        {
            processStatus.Items.Add(tip);

        }

        #endregion

        /// <summary>
        /// 确认类型和标签已经被加载
        /// </summary>
        void EnsureModel()
        {
            if (model != null) return;

            model = new Model();

            #region 加载类型及标签

            ProcessTip("加载电影类型");
            Data.List.MovieGenresClass mtClass = new Data.List.MovieGenresClass { RowCount = int.MaxValue };
            var mvtags = mtClass.GetArray();
            model.LoadProductGenres(mvtags);
            ProcessTip("加载完毕，共有" + mvtags.Length + "条记录");
            mvtags = null;

            //ProcessTip("加载电影标签");
            //Data.List.MovieTagClass tgClass = new Data.List.MovieTagClass { RowCount = int.MaxValue };
            //实际上，如果某个标签只出现过一次，那么这个标签就失去了特征意义。

            //var tgs = tgClass.GetArray();
            //model.LoadProductTags(tgs);
            //ProcessTip("完毕，共有" + tgs.Length + "条记录");
            //tgs = null;

            #endregion

        }

        void ProcessNew()
        {
            //重置
            model = null;
            EnsureModel();

            ProcessTip("开始处理");
            //从最小值开始每次增加一进行处理
            Data.List.RatingHistoryLiteClass rhClass = new Data.List.RatingHistoryLiteClass();

            int lastTick = 0;
            for (int tick = 830; tick < 834; tick++)
            {
                rhClass.TimeStampStart = lastTick * tickStep + 1;
                //rhClass.TimeStampEnd = tick * tickStep;
                rhClass.RowCount = int.MaxValue;
                lastTick = tick;

                ProcessTip("新：" + tick);

                var hisdata = rhClass.GetArray();

                ProcessTip("记录数：" + hisdata.Length);

                int splitindex = (int)(hisdata.Length * 0.8);
                //80-20分割验证

                model.RatesForForest = hisdata;
                var processData = hisdata.Take(splitindex);
                //var processData = hisdata.AsEnumerable();
                var forestData = hisdata.Skip(splitindex);

                ProcessTip(string.Format("处理数据{0}条，预测数据{1}条", splitindex, forestData.Count()));

                int scantime = 0;

                while (scantime++ < 1)
                {
                    //读取一段新数据并进行处理（十次）
                    //for (int i = 0; i < 10; i++)
                    {
                        model.ProcessRate(processData);
                    }

                    #region 读取投票数据进行验证

                    //rhClass.TimeStampStart = (tick - 1) * tickStep + 1;
                    //rhClass.TimeStampEnd = 0;
                    //rhClass.RowCount = 1000;
                    //var hisData = rhClass.GetArray();
                    int wu0 = 0;
                    int wu1 = 0;
                    int wu2 = 0;

                    //MSE计算公式
                    //平方差之和除以N
                    int mse = 0;
                    int mae = 0;
                    int total = 0;

                    int nonewmse = 0;
                    int nonewmae = 0;
                    int nonewtotal = 0;

                    foreach (var h in forestData)
                    {
                        var result = model.ForestRating(h.UserID, h.MovieID);
                        int foreScore = result.Rating;
                        
                        total++;
                        //2.检查是否命中
                        int wucha = Math.Abs((int)h.Rating - foreScore);

                        if (wucha == 0) wu0++;
                        else if (wucha == 1) wu1++;
                        else if (wucha == 2) wu2++;

                        mse += wucha * wucha;
                        mae += wucha;

                        if (!result.IsNewUser)
                        {
                            nonewtotal++;
                            nonewmse += wucha * wucha;
                            nonewmae += wucha;
                        }
                    }

                    //ProcessTip(string.Format("新用户{0}次，老用户{1}次", newUser, total));
                    ProcessTip(string.Format("第{7}遍，总计{0}，其中误差0={1}，误差0~1={2}({3}%)，误差0~2({6})，MSE值{4}，MAE值{5}", total, wu0, wu0 + wu1
                        , total == 0 ? 0 : (int)((wu0 + wu1) * 100 / total)
                        , total == 0 ? 0d : ((double)mse / total)
                        , total == 0 ? 0d : ((double)mae / total)
                        , wu0 + wu1 + wu2
                        , scantime// * 10
                    ));

                    ProcessTip(string.Format("第{3}遍，老用户总计{0}，MSE值{1}，MAE值{2}", nonewtotal
                        , nonewtotal == 0 ? 0d : ((double)nonewmse / nonewtotal)
                        , nonewtotal == 0 ? 0d : ((double)nonewmae / nonewtotal)
                        , scantime// * 10
                    ));


                    #endregion
                }
            }

            ProcessTip("处理完毕");

        }

        /// <summary>
        /// 处理并实时显示过程
        /// </summary>
        void ProcessAll()
        {
            //首先加载标签数据
            //2.准备电影标签初始数据
            Dictionary<long, UserData> users = new Dictionary<long, UserData>();
            Dictionary<long, MovieData> movies = new Dictionary<long, MovieData>();

            #region 加载类型及标签

            ProcessTip("加载电影类型");
            ProcessTip("加载完毕，共有" + LoadMovieGenres(movies) + "条记录");

            //ProcessTip("加载电影标签");
            //ProcessTip("加载完毕，共有" + LoadMovieTags(movies) + "条记录");

            #endregion

            ProcessTip("开始处理");
            //从最小值开始每次增加一进行处理
            Data.List.RatingHistory rhClass = new Data.List.RatingHistory();

            int lastTick = 0;
            for (int tick = 830; tick < 850; tick++)
            {
                rhClass.TimeStampStart = lastTick * tickStep + 1;
                rhClass.TimeStampEnd = tick * tickStep;
                rhClass.RowCount = int.MaxValue;
                lastTick = tick;

                ProcessTip("新：" + tick);

                #region 读取一段新数据并进行处理
                var hisdata = rhClass.GetArray();
                foreach (var his in hisdata)
                {
                    MovieData md = movies[his.MovieID];
                    UserData ud;
                    long uid = his.UserID;
                    if (!users.TryGetValue(uid, out ud))
                    {
                        ud = new UserData();
                        users.Add(uid, ud);
                    }
                    int rating = ((int)his.Rating) - 4;//引入负分机制

                    //3.1先对用户处理
                    ud.RateMovie(md, rating);

                    //3.2再对电影处理
                    md.BeRated(ud, rating);
                }
                hisdata = null;
                GC.Collect();
                #endregion

                #region 读取投票数据进行验证

                rhClass.TimeStampStart = (tick - 1) * tickStep + 1;
                rhClass.TimeStampEnd = 0;
                rhClass.RowCount = 1000;
                var hisData = rhClass.GetArray();
                int total = 0;
                int wu0 = 0;
                int wu1 = 0;
                //int wu2 = 0;
                int newUser = 0;

                //MSE计算公式
                //平方差之和除以N
                int mse = 0;
                int mae = 0;

                foreach (var h in hisData)
                {
                    //1.预测
                    MovieData md = movies[h.MovieID];

                    if (!users.ContainsKey(h.UserID))
                    {
                        #region 新用户处理
                        newUser++;
                        //UserData ud = new UserData();
                        //users.Add(h.UserID, ud);
                        #endregion
                    }
                    else
                    {
                        #region 已有数据预测
                        total++;
                        UserData ud = users[h.UserID];

                        var value = ForecastRating(ud, md, false);
                        int match = value.match;
                        int foreScore = value.score;

                        //2.检查是否命中
                        int wucha = (int)h.Rating - foreScore;

                        if (wucha == 0) wu0++;
                        else if (wucha == 1) wu1++;
                        //else if (wucha == 2) wu2++;

                        mse += wucha * wucha;
                        mae += wucha;

                        #endregion
                    }
                }

                ProcessTip(string.Format("新用户{0}次，老用户{1}次", newUser, total));
                ProcessTip(string.Format("总计{0}，其中误差0={1}，误差0~1={2}({3}%)，MSE值{4}，MAE值{5}", total, wu0, wu0 + wu1
                    , total == 0 ? 0 : (int)((wu0 + wu1) * 100 / total)
                    , total == 0 ? 0d : ((double)mse / total)
                    , total == 0 ? 0d : ((double)mae / total)
                ));

                #endregion

                if (allowForget)
                {

                    //遗忘处理
                    foreach (var ud in users.Values)
                    {
                        ud.Foreget();
                    }

                    foreach (var md in movies.Values)
                    {
                        md.Foreget();
                    }

                }
            }
        }

        #region 加载标签和类别

        private static int LoadMovieGenres(Dictionary<long, MovieData> movies)
        {
            Data.List.MovieGenresClass mtClass = new Data.List.MovieGenresClass { RowCount = int.MaxValue };
            var mvtags = mtClass.GetArray();

            //SetProgressMax(mvtags.Length);
            foreach (var mvt in mvtags)
            {
                long mid = mvt.MovieID;
                MovieData md;
                if (!movies.TryGetValue(mid, out md))
                {
                    md = new MovieData();
                    movies.Add(mid, md);
                    md.BeginProcessTag();
                }
                md.ProcessTag(mvt.TagID, 5000, 5000);
                //IncreaseProgress();
            }
            return mvtags.Length;
        }

        private static int LoadMovieTags(Dictionary<long, MovieData> movies)
        {
            Data.List.MovieTagClass mtagClass = new Data.List.MovieTagClass { RowCount = int.MaxValue };
            var tags = mtagClass.GetArray();
            //SetProgressMax(tags.Length);
            foreach (var t in tags)
            {
                long mid = t.MovieID;
                MovieData md;
                if (!movies.TryGetValue(mid, out md))
                {
                    md = new MovieData();
                    movies.Add(mid, md);
                    md.BeginProcessTag();
                }
                md.ProcessTag(t.TagID, 3000, 3000);

                //IncreaseProgress();
            }

            foreach (MovieData md in movies.Values)
            {
                md.EndProcessTag();
            }

            return tags.Length;
        }

        #endregion

        #region 未用

        /// <summary>
        /// 一个标签信息
        /// </summary>
        struct TagInfo : IComparable
        {
            public long TagID;
            public int Level;
            /// <summary>
            /// 出现的次数
            /// </summary>
            public int Count;
            /// <summary>
            /// 总评分
            /// </summary>
            public int SumScore;
            /// <summary>
            /// 上次时间点总评分
            /// </summary>
            public int PrevSumScore;

            public override int GetHashCode()
            {
                return TagID.GetHashCode();
            }

            public override bool Equals(object obj)
            {
                return ((TagInfo)obj).TagID == TagID;
            }

            #region IComparable 成员

            public int CompareTo(object obj)
            {
                return Level - ((TagInfo)obj).Level;
            }

            #endregion
        }
        #endregion
    }
}
